/*

mydownlib
by Luigi Auriemma
e-mail: aluigi@autistici.org
web:    aluigi.org

    Copyright 2006,2007 Luigi Auriemma

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA

    http://www.gnu.org/licenses/gpl.txt
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>
#include <sys/stat.h>
#include <time.h>
#include <ctype.h>
#include <zlib.h>
#include "mydownlib.h"

#ifdef WIN32
    #include <winsock.h>

    #define close       closesocket
    #define in_addr_t   uint32_t
    #define TEMPOZ1
    #define TEMPOZ2     GetTickCount()
    #define ONESEC      1000
#else
    #include <unistd.h>
    #include <sys/socket.h>
    #include <sys/types.h>
    #include <arpa/inet.h>
    #include <netinet/in.h>
    #include <netdb.h>
    #include <sys/times.h>
    #include <sys/timeb.h>

    #define stristr     strcasestr
    #define stricmp     strcasecmp
    #define strnicmp    strncasecmp
    #define TEMPOZ1     ftime(&timex)
    #define TEMPOZ2     ((timex.time * 1000) + timex.millitm)
    #define ONESEC      1
#endif



#define VISDELAY    500
#define BUFFSZ      4096
#define MAXARGS     8           // modify it if you need more args in mydown_scanhead
#define MAXDNS      32
#define TEMPOZ(x)   TEMPOZ1;    \
                    x = TEMPOZ2



typedef struct {                // lame DNS caching implementation
    uint8_t     *host;
    in_addr_t   ip;
} dns_db_t;

int         dns_db_max      = 0,
            dns_db_add      = 0;
dns_db_t    dns_db[MAXDNS];



void mydown_get_host(uint8_t *url, uint8_t **hostx, uint16_t *portx, uint8_t **urix, uint8_t **userx, uint8_t **passx, int verbose);
uint8_t *mydown_http_delimit(uint8_t *data);
uint8_t *mydown_uri2hex(uint8_t *uri);
uint8_t *mydown_hex2uri(uint8_t *uri);
void mydown_scanhead(uint8_t *data, int datalen, ...);
uint8_t *mydown_http_skip(uint8_t *buff, int len, int *needed, int *remain);
void mydown_free(uint8_t **buff);
int mydown_chunked_skip(uint8_t *buff, int chunkedsize);
int mydown_unzip(z_stream z, uint8_t *in, int inlen, uint8_t **outx, int *outxlen);
int mydown_sscanf_hex(uint8_t *data, int datalen);
int mydown_timeout(int sock, int secs);
int mydown_recv(int sd, uint8_t *data, int len, int timeout);
uint8_t *mydown_showhttp80(uint16_t port);
void mydown_showstatus(uint32_t fsize, uint32_t ret, int timediff, int verbose);
uint8_t *mydown_base64_encode(uint8_t *data, int *length);
in_addr_t mydown_resolv(char *host);



uint32_t mydown(uint8_t *myurl, uint8_t *filename, mydown_options *opt) {
    FILE        *fd          = NULL;
    uint32_t    from         = 0,
                tot          = 0,
                filesize     = MYDOWN_ERROR;
    int         showhead     = 0,
                resume       = 0,
                onlyifdiff   = 0,
                verbose      = 0,
                *keep_alive  = 0,
                *ret_code    = 0,
                timeout      = 0,
                onflyunzip   = 0,
                contentsize  = 0;
    uint16_t    port         = 0;
    uint8_t     *url         = NULL,
                *uri         = NULL,
                *host        = NULL,
                *user        = NULL,
                *pass        = NULL,
                *referer     = NULL,
                *useragent   = NULL,
                *cookie      = NULL,
                *more_http   = NULL,
                **filedata   = NULL,
                *content     = NULL,
                *get         = NULL;

#ifdef WIN32
    WSADATA wsadata;
    WSAStartup(MAKEWORD(1,0), &wsadata);
#endif

    setbuf(stdout, NULL);
    setbuf(stderr, NULL);

    if(opt) {
        from         = opt->from;
        tot          = opt->tot;
        showhead     = opt->showhead;
        resume       = opt->resume;
        onlyifdiff   = opt->onlyifdiff;
        user         = opt->user;
        pass         = opt->pass;
        referer      = opt->referer;
        useragent    = opt->useragent;
        cookie       = opt->cookie;
        more_http    = opt->more_http;
        verbose      = opt->verbose;
        filedata     = opt->filedata;
        keep_alive   = opt->keep_alive;
        timeout      = opt->timeout;
        ret_code     = opt->ret_code;
        onflyunzip   = opt->onflyunzip;
        content      = opt->content;
        contentsize  = opt->contentsize;
        get          = opt->get;

        if(user) {
            if(verbose > 0) fprintf(stderr,
                "  user   %s\n"
                "  pass   %s\n",
                user,
                pass);
        }
    }

    if(!myurl) return(MYDOWN_ERROR);
    url = strdup(myurl);
    if(!url) return(MYDOWN_ERROR);
    mydown_get_host(url, &host, &port, &uri, &user, &pass, verbose);

    if(verbose > 0) fprintf(stderr, "  start download\n");

    filesize = mydown_http2file(
        keep_alive,         // int *sock
        timeout,            // int timeout
        host,               // uint8_t *host
        port,               // uint16_t port
        user,               // uint8_t *user
        pass,               // uint8_t *pass
        referer,            // uint8_t *referer
        useragent,          // uint8_t *useragent
        cookie,             // uint8_t *cookie
        more_http,          // uint8_t *more_http
        verbose,            // int verbose
        uri,                // uint8_t *getstr
        fd,                 // FILE *fd
        filename,           // uint8_t *filename
        showhead,           // int showhead
        onlyifdiff,         // int onlyifdiff
        resume,             // int resume
        from,               // uint32_t from
        tot,                // uint32_t tot
        NULL,               // uint32_t *filesize
        filedata,           // uint8_t **filedata
        ret_code,           // int *ret_code
        onflyunzip,         // int onflyunzip
        content,            // uint8_t *content
        contentsize,        // int contentsize
        get                 // uint8_t *get
    );

    if(fd)  fclose(fd);
    if(url) free(url);
    if(uri) free(uri);
    return(filesize);
}



void mydown_get_host(uint8_t *url, uint8_t **hostx, uint16_t *portx, uint8_t **urix, uint8_t **userx, uint8_t **passx, int verbose) {
    uint16_t    port  = 80;
    uint8_t     *host = NULL,
                *uri  = NULL,
                *user = NULL,
                *pass = NULL,
                *p;

    host = url;

    p = strstr(host, "://");    // handle http://
    if(!p) p = strstr(host, ":\\\\");
    if(p) {
        for(p += 3; *p; p++) {  // in case of http:////
            if((*p != '/') && (*p != '\\')) break;
        }
        host = p;
    }

    for(p = host; *p; p++) {    // search the uri
        if((*p == '/') || (*p == '\\')) {
            uri = p;
            break;
        }
    }
    if(uri) {
        *uri++ = 0;
        uri = mydown_uri2hex(uri);
    }
    if(!uri) uri = strdup("");  // in case mydown_uri2hex fails

    p = strrchr(host, '@');
    if(p) {
        *p = 0;

        user = host;

        pass = strchr(host, ':');
        if(pass) {
            *pass++ = 0;
        } else {
            pass = "";
        }

        host = p + 1;
    }

    p = strchr(host, ':');
    if(p) {
        *p = 0;
        port = atoi(p + 1);
    }

    if(verbose >= 0) fprintf(stderr, "  %s\n", url);
    if(verbose > 0) fprintf(stderr,
        "  host   %s : %hu\n"
        "  uri    %s\n",
        host, port,
        uri);

    if(user) {
        if(verbose > 0) fprintf(stderr,
            "  user   %s\n"
            "  pass   %s\n",
            user,
            pass);
    }

    *hostx = host;
    *portx = port;
    *urix  = uri;
    *userx = user;
    *passx = pass;
}



uint8_t *mydown_http_delimit(uint8_t *data) {
    if(!data || !data[0]) return(NULL);
    while(*data && (*data != '\r') && (*data != '\n')) data++;
    *data = 0;
    for(data++; *data && ((*data == '\r') || (*data == '\n')); data++);
    return(data);
}



uint8_t *mydown_uri2hex(uint8_t *uri) {
    static const uint8_t hex[16] = "0123456789abcdef";
    uint8_t     *ret,
                *p,
                c;

    ret = malloc((strlen(uri) * 3) + 1);
    if(!ret) return(NULL);

    for(p = ret; *uri; uri++) {
        c = *uri;
        if(isprint(c)) {
            *p++ = c;
        } else {
            *p++ = '%';
            *p++ = hex[c >> 4];
            *p++ = hex[c & 15];
        }
    }
    *p = 0;

    return(ret);
}



uint8_t *mydown_hex2uri(uint8_t *uri) {
    int         t;
    uint8_t     *ret,
                *p;

    ret = strdup(uri);
    if(!ret) return(NULL);

    for(p = ret; *uri; uri++, p++) {
        if(*uri == '%') {
            sscanf(uri + 1, "%02x", &t);
            uri += 2;
            *p = t;
        } else {
            *p = *uri;
        }
    }
    *p = 0;

    return(ret);
}



void mydown_scanhead(uint8_t *data, int datalen, ...) {
    va_list     ap;
    int         i,
                vals;
    uint8_t     *par[MAXARGS],
                **val[MAXARGS],
                *l,
                *p,
                *limit;

    va_start(ap, datalen);
    for(i = 0; i < MAXARGS; i++) {
        par[i] = va_arg(ap, uint8_t *);
        if(!par[i]) break;
        val[i] = va_arg(ap, uint8_t **);
        if(!val[i]) break;
        *val[i] = NULL;
    }
    vals = i;
    va_end(ap);

    for(limit = data + datalen; (l = mydown_http_delimit(data)); data = l) {
        if(l > limit) break;
        p = strchr(data, ':');
        if(!p) continue;
        *p++ = 0;
        for(i = 0; i < vals; i++) {
            if(stricmp(data, par[i])) continue;
            while(*p && ((*p == ' ') || (*p == '\t'))) p++;
            *val[i] = p;
            break;
        }
    }
}



uint32_t mydown_http2file(int *sock, int timeout, uint8_t *host, uint16_t port, uint8_t *user, uint8_t *pass, uint8_t *referer, uint8_t *useragent, uint8_t *cookie, uint8_t *more_http, int verbose, uint8_t *getstr, FILE *fd, uint8_t *filename, int showhead, int onlyifdiff, int resume, uint32_t from, uint32_t tot, uint32_t *filesize, uint8_t **filedata, int *ret_code, int onflyunzip, uint8_t *content, int contentsize, uint8_t *get) {
#ifndef WIN32
    struct  timeb   timex;
#endif
    z_stream    z;
    struct  sockaddr_in peer;
    struct      stat    xstat;
    time_t      oldtime         = 0,
                newtime;
    uint32_t    ret             = 0,
                httpret         = 0,
                fsize           = 0;
    int         sd              = 0,
                t,
                err,
                len,
                code            = 0,
                b64len,
                filedatasz      = 0,
                httpcompress    = 0,
                httpgzip        = 0,
                httpdeflate     = 0,
                httpz           = 0,
                chunked         = 0,
                chunkedsize     = 0,
                chunkedlen      = 0,
                wbits,
                zbufflen        = 0,
                httpskipbytes   = 0;
    uint8_t     *buff           = NULL,
                *query          = NULL,
                *data           = NULL,
                *p              = NULL,
                *s              = NULL,
                *userpass       = NULL,
                *b64            = NULL,
                *conttype       = NULL,
                *contlen        = NULL,
                *contdisp       = NULL,
                *icyname        = NULL,
                *transenc       = NULL,
                *contenc        = NULL,
                *location       = NULL,
                *filedatatmp    = NULL,
                *zbuff          = NULL,
                *ztmp           = NULL,
                *chunkedbuff    = NULL,
                *chunkedtmp     = NULL,
                *filenamemalloc = NULL;

#define GOTOQUIT    { ret = MYDOWN_ERROR; goto quit; }

    if(!sock || (sock && !*sock)) {
        peer.sin_addr.s_addr = mydown_resolv(host);
        if(!peer.sin_addr.s_addr) GOTOQUIT;
        peer.sin_port        = htons(port);
        peer.sin_family      = AF_INET;

        sd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
        if(sd < 0) GOTOQUIT;
        if(sock) *sock = sd;

        if(verbose > 0) fprintf(stderr, "  connect to %s:%hu...", inet_ntoa(peer.sin_addr), port);
        if(connect(sd, (struct sockaddr *)&peer, sizeof(peer)) < 0) {
            fprintf(stderr, "\nError: connection refused\n");
            GOTOQUIT;
        }
        if(verbose > 0) fprintf(stderr, "done\n");
    } else {
        sd = *sock;
    }

    if(user && pass) {
        userpass = malloc(strlen(user) + 1 + strlen(pass) + 1);
        if(!userpass) GOTOQUIT;
        b64len = sprintf(userpass, "%s:%s", user, pass);
        b64 = mydown_base64_encode(userpass, &b64len);
    }

    if(!get) get = "GET";
    if(content && !contentsize) contentsize = strlen(content);

    len =
        200                                       + // my format strings
        strlen(get)                               +
        strlen(getstr)                            +
        strlen(host)                              +
        5                                         +
        ((from || tot)   ? 10 + 10           : 0) +
        (b64             ? strlen(b64)       : 0) +
        (referer         ? strlen(referer)   : 0) +
        (useragent       ? strlen(useragent) : 0) +
        (cookie          ? strlen(cookie)    : 0) +
        (more_http       ? strlen(more_http) : 0) +
        (content         ? contentsize       : 0) +
        1;

    query = malloc(len);
    if(!query) GOTOQUIT;

    len = sprintf(
        query,
        "%s /%s HTTP/1.1\r\n"
        "Host: %s%s\r\n"
        "Connection: %s\r\n",
        get, getstr,
        host, mydown_showhttp80(port),
        sock ? "keep-alive" : "close");

#define HTTP_APPEND  len += sprintf(query + len,

    if(!onlyifdiff) {   // x-compress needs unlzw???
        HTTP_APPEND "Accept-Encoding: deflate,gzip,x-gzip,compress,x-compress\r\n");
    }
    if(from || tot) {
        HTTP_APPEND "Range: bytes=");
        if(from != -1) HTTP_APPEND "%u", from);
        HTTP_APPEND "-");
        if(tot)        HTTP_APPEND "%u", tot + ((from == -1) ? 0 : from));
        HTTP_APPEND "\r\n");
    }
    if(b64) {
        HTTP_APPEND "Authorization: Basic %s\r\n", b64);
    }
    if(referer) {
        HTTP_APPEND "Referer: %s\r\n", referer);
    }
    if(useragent) {
        HTTP_APPEND "User-Agent: %s\r\n", useragent);
    }
    if(cookie) {
        HTTP_APPEND "Cookie: %s\r\n", cookie);
    }
    if(more_http) {
        HTTP_APPEND "%s", more_http);
        if(query[len - 1] == '\r') {
            HTTP_APPEND "\n");
        } else if(query[len - 1] != '\n') {
            HTTP_APPEND "\r\n");
        }
    }
    if(content) {
        HTTP_APPEND "Content-length: %d\r\n", contentsize);
    }
    HTTP_APPEND "\r\n");
    if(content) {
        memcpy(query + len, content, contentsize);
        len += contentsize;
    }

#undef HTTP_APPEND  

    send(sd, query, len, 0);
    mydown_free(&query);

    buff = malloc(BUFFSZ + 1);
    if(!buff) GOTOQUIT;

    data = p = buff;
    len  = BUFFSZ;
    while((t = mydown_recv(sd, data, len, timeout)) > 0) {
        data += t;
        len  -= t;
        *data = 0;

        p = strstr(buff, "\r\n\r\n");
        if(p) {
            p += 4;
        } else {
            p = strstr(buff, "\n\n");
            if(p) p += 2;
        }

        if(p) {
            *(p - 1) = 0;

            if(showhead) {
                fprintf(stderr, "\n%s", buff);
                goto quit;
            }

            s = strchr(buff, ' ');
            if(s) {
                code = atoi(s + 1);

                if((code / 100) == 3) {
                    mydown_scanhead(buff, p - buff,
                        "location",     &location,
                        NULL,           NULL);
                    if(!location) {
                        fprintf(stderr, "\nError: remote file is temporary unavailable (%d)\n", code);
                        GOTOQUIT;
                    }
                    if(verbose > 0) fprintf(stderr, "\n- redirect: %s\n", location);
                    mydown_get_host(location, &host, &port, &getstr, &user, &pass, verbose);
                    ret = mydown_http2file(sock, timeout, host, port, user, pass, referer, useragent, cookie, more_http, verbose, getstr, fd, filename, showhead, onlyifdiff, resume, from, tot, filesize, filedata, ret_code, onflyunzip, content, contentsize, get);
                    goto quit;
                }

                    // if((code != 200) && (code != 206)) {
                if((code / 100) != 2) {
                    fprintf(stderr, "\nError: remote file is temporary unavailable (%d)\n", code);
                    GOTOQUIT;
                }
            }

            mydown_scanhead(buff, p - buff,
                "content-length",       &contlen,
                "content-type",         &conttype,
                "content-disposition",  &contdisp,
                "icy-name",             &icyname,
                "transfer-encoding",    &transenc,
                "content-encoding",     &contenc,
                "Set-Cookie",           &cookie,
                NULL,                   NULL);

            if(contlen) {
                s = strchr(contlen, '/');
                if(s) contlen = s + 1;
                sscanf(contlen, "%u", &fsize);
            }

            if(conttype) {
                if(stristr(conttype, "compress"))   httpcompress  = 1;
                if(stristr(conttype, "gzip"))       httpgzip      = 1;
                if(!onflyunzip) if(stristr(conttype, "x-gzip"))     httpgzip      = 0;  // work-around
                if(stristr(conttype, "deflate"))    httpdeflate   = 1;
            }

            if(contenc) {
                if(stristr(contenc,  "compress"))   httpcompress  = 1;
                if(stristr(contenc,  "gzip"))       httpgzip      = 1;
                if(!onflyunzip) if(stristr(contenc,  "x-gzip"))     httpgzip      = 0;  // work-around
                if(stristr(contenc,  "deflate"))    httpdeflate   = 1;
            }

            if(!contdisp && icyname) contdisp = icyname;

            if(transenc) {
                if(stristr(transenc, "chunked"))    chunked = 1;
            }

            if(!filename || !filename[0]) {
                if(contdisp) {
                    s = (uint8_t *)stristr(contdisp, "filename=");
                    if(!s) s = (uint8_t *)stristr(contdisp, "file=");
                    if(s) {
                        s = strchr(s, '=') + 1;
                    } else {
                        s = contdisp;
                    }
                    while(*s && ((*s == '\"') || (*s == ' ') || (*s == '\t'))) s++;
                    filename = mydown_hex2uri(s);
                    if(filename) filenamemalloc = filename; // needed for freeing it later!
                    if(filename && filename[0]) {
                        for(s = filename; *s; s++) {
                            if((*s == '\\') || (*s == '/') || (*s == ';') || (*s == ':') || (*s == '\"') || (*s == '&') || (*s == '?')) break;
                        }
                        for(s--; (s >= filename) && *s && ((*s == ' ') || (*s == '\t')); s--);
                        *(s + 1) = 0;
                    }

                } else {
                    filename = mydown_hex2uri(getstr);
                    if(filename) filenamemalloc = filename; // needed for freeing it later!
                    if(filename && filename[0]) {
                        for(
                          s = filename + strlen(filename) - 1;
                          (s >= filename) && (*s != '/') && (*s != '\\') && (*s != ':') && (*s != '&') && (*s != '?') && (*s != '=');
                          s--);
                        filename = s + 1;
                    }
                }

                if(filename) {                          // last useless check to avoid directory traversal
                    s = strrchr(filename, ':');
                    if(s) filename = s + 1;
                    s = strrchr(filename, '\\');
                    if(s) filename = s + 1;
                    s = strrchr(filename, '/');
                    if(s) filename = s + 1;

                    s = strrchr(filename, '.');         // automatic gzip decompression on the fly
                    if(s && !stricmp(s, ".gz")) {
                        if(onflyunzip) {
                            httpgzip = 1;
                            *s = 0;
                        }
                    }
                }
            }

            if(!filename || !filename[0]) {
                if(verbose >= 0) fprintf(stderr, "\nError: no filename retrieved, you must specify an output filename\n\n");
                if(filesize) *filesize = MYDOWN_ERROR;
                GOTOQUIT;
            }

            if(!filedata && !fd) {
                if(!strcmp(filename, "-")) {
                    fd = stdout;
                    if(verbose >= 0) fprintf(stderr, "  file   %s\n", "stdout");
                } else {
                    if(verbose >= 0) fprintf(stderr, "  file   %s\n", filename);
                    err = stat(filename, &xstat);
                    if(onlyifdiff && !err && (xstat.st_size == fsize)) {
                        if(verbose >= 0) fprintf(stderr, "  the remote file has the same size of the local one, skip\n");
                        if(filesize) *filesize = MYDOWN_ERROR;
                        GOTOQUIT;
                    }
                    if((err < 0) || !resume) {      // file doesn't exist and must not resume
                        fd = fopen(filename, "wb");
                    } else {
                        fd = fopen(filename, "ab");
                        from = xstat.st_size;
                        if(verbose > 0) fprintf(stderr, "  resume %u\n", from);
                        ret = mydown_http2file(sock, timeout, host, port, user, pass, referer, useragent, cookie, more_http, verbose, getstr, fd, filename, showhead, onlyifdiff, resume, from, tot, filesize, filedata, ret_code, onflyunzip, content, contentsize, get);
                        goto quit;
                    }
                    if(!fd) {
                        if(filesize) *filesize = MYDOWN_ERROR;
                        GOTOQUIT;
                    }
                }
            }

            break;
        }
    }

    if(t < 0) GOTOQUIT;

    if(!p) p = buff;
    len = data - p;
    memmove(buff, p, len);

    httpz = 1;
    if(httpcompress) {
        if(verbose > 0) fprintf(stderr, "  compression: compress\n");
        wbits =  15;
    } else if(httpgzip) {
        if(verbose > 0) fprintf(stderr, "  compression: gzip\n");
        wbits = -15;
    } else if(httpdeflate) {
        if(verbose > 0) fprintf(stderr, "  compression: deflate\n");
        wbits = -15;
    } else {
        httpz = 0;
    }
    if(httpz) {
        z.zalloc = (alloc_func)0;
        z.zfree  = (free_func)0;
        z.opaque = (voidpf)0;
        if(inflateInit2(&z, wbits)) GOTOQUIT;

        zbufflen = BUFFSZ * 4;
        zbuff    = malloc(zbufflen);
        if(!zbuff) GOTOQUIT;
    }

    if(verbose > 0) fprintf(stderr, "\n");
    if(fsize) if(verbose > 0) fprintf(stderr, "    ");
    if(verbose > 0) fprintf(stderr, " | downloaded | bytes/second\n");
    if(fsize) if(verbose > 0) fprintf(stderr, "----");
    if(verbose > 0) fprintf(stderr, "-/------------/-------------\n");

    TEMPOZ(oldtime);
    oldtime -= VISDELAY;

    if(filedata) {
        filedatasz  = fsize;
        filedatatmp = malloc(filedatasz);
        if(!filedatatmp) GOTOQUIT;
    }

    if(chunked) chunkedsize = len;

    do {
redo:
        httpret += len;

        if(chunked) {
            for(;;) {
                chunkedsize = mydown_chunked_skip(buff, chunkedsize);

                err = mydown_sscanf_hex(buff, chunkedsize);
                if(err > 0) break;
                if(!err) {
                    chunkedsize = mydown_chunked_skip(buff, chunkedsize);
                    break;
                }

                t = mydown_recv(sd, buff + chunkedsize, BUFFSZ - chunkedsize, timeout);
                if(t <= 0) GOTOQUIT;
                chunkedsize += t;
                if(chunkedsize >= BUFFSZ) GOTOQUIT;
            }

            chunkedlen = err;
            if(!chunkedlen) break;

            if(chunkedbuff) free(chunkedbuff);
            chunkedbuff = malloc(chunkedlen);
            if(!chunkedbuff) GOTOQUIT;

            err = ((uint8_t *)strchr(buff, '\n') + 1) - buff;
            chunkedsize -= err;
            memmove(buff, buff + err, chunkedsize);

            if(chunkedlen < chunkedsize) {      // we have more data than how much we need
                memcpy(chunkedbuff, buff, chunkedlen);
                chunkedsize -= chunkedlen;
                memmove(buff, buff + chunkedlen, chunkedsize);
            } else {                            // we have only part of the needed data
                memcpy(chunkedbuff, buff, chunkedsize);
                for(len = chunkedsize; len < chunkedlen; len += t) {
                    t = mydown_recv(sd, chunkedbuff + len, chunkedlen - len, timeout);
                    if(t <= 0) GOTOQUIT;
                }
                chunkedsize = 0;
            }

            chunkedtmp  = buff;
            buff        = chunkedbuff;
            len         = chunkedlen;
        }

            /* DECOMPRESSION */

        if(httpz) {
            if(httpgzip && !ret) {  // gzip is really stupid...
                t = len;
                s = buff;
                if((httpgzip == 1) && !httpskipbytes) httpskipbytes = 3;

                for(;;) {
                    s = mydown_http_skip(s, t, &httpskipbytes, &t);
                    if(!s) {
                        t = len;
                        break;
                    }
                    if(httpgzip == 1) {
                        httpgzip = (*s) ? 2 : 4;
                        httpskipbytes = 7;
                    } else if(httpgzip == 2) {  // timestamp and name
                        httpgzip = 3;
                    } else if(httpgzip == 3) {
                        while((s - buff) < len) {
                            if(!*s++) {
                                httpgzip = 4;
                                break;
                            }
                        }
                    } else {
                        t = s - buff;
                        break;
                    }
                }
                len = mydown_unzip(z, buff + t, len - t, &zbuff, &zbufflen);
            } else {
                len = mydown_unzip(z, buff, len, &zbuff, &zbufflen);
            }
            if(len < 0) GOTOQUIT;
            ztmp = buff;
            buff = zbuff;
        }

            /* UPDATE THE AMOUNT OF UNCOMPRESSED BYTES DOWNLOADED */
            // ret is the total size of the data we have downloaded (uncompressed)
            // httpret is the total size of the data we have downloaded from the server
            // len is the size of the current block of data we have downloaded (uncompressed)

        ret += len;

            /* WRITE THE DATA INTO FILE OR MEMORY */

        if(filedata) {
            if(filedatasz < ret) {
                filedatasz  = ret;
                filedatatmp = realloc(filedatatmp, filedatasz);
                if(!filedatatmp) GOTOQUIT;
            }
            memcpy(filedatatmp + ret - len, buff, len);
        } else if(fd) {
            if(fwrite(buff, 1, len, fd) != len) {
                fprintf(stderr, "\nError: I/O error. Probably your disk is full or the file is write protected\n");
                GOTOQUIT;
            }
            fflush(fd);
        }

            /* VISUALIZATION */

        TEMPOZ(newtime);
        if((newtime - oldtime) >= VISDELAY) {
            mydown_showstatus(fsize, httpret, (int)(newtime - oldtime), verbose);
            oldtime = newtime;
        }

            /* FREE, EXCHANGE OR OTHER STUFF */

        if(httpz) {
            zbuff = buff;
            buff  = ztmp;
        }
        if(chunked) {
            chunkedbuff = buff;
            buff        = chunkedtmp;
            len         = 0;
            goto redo;
        }

            /* FSIZE CHECK */

        if(fsize) {
            if(httpret >= fsize) break;
        }

            /* READ NEW DATA FROM THE STREAM */

    } while((len = mydown_recv(sd, buff, BUFFSZ, timeout)) > 0);

    TEMPOZ(newtime);
    mydown_showstatus(fsize, httpret, (int)(newtime - oldtime), verbose);

    if(fsize && (len < 0)) GOTOQUIT;

    if(filedata) {
        *filedata = filedatatmp;
    }

quit:
    if(httpz) {
        if(zbuff) inflateEnd(&z);
        if(zbuff != buff) mydown_free(&zbuff);
    }
    if(chunkedbuff != buff) mydown_free(&chunkedbuff);
    mydown_free(&userpass);
    mydown_free(&b64);
    mydown_free(&buff);
    mydown_free(&filenamemalloc);
    if(ret_code) *ret_code = code;
    if(sd && !sock) close(sd);
    if(verbose >= 0) fputc('\n', stderr);
    if(ret == MYDOWN_ERROR) {
        if(sock && *sock) {
            close(*sock);
            *sock = 0;
        }
    }
    return(ret);

#undef GOTOQUIT
}



uint8_t *mydown_http_skip(uint8_t *buff, int len, int *needed, int *remain) {
    int     rest;

    rest = *needed;
    if(len < rest) {
        *needed = rest - len;
        *remain = 0;
        return(NULL);
    }
    *needed = 0;
    *remain = len - rest;
    return(buff + rest);
}



void mydown_free(uint8_t **buff) {
    if(!*buff) return;
    free(*buff);
    *buff = NULL;
}



int mydown_chunked_skip(uint8_t *buff, int chunkedsize) {
    int         t;

    for(t = 0; t < chunkedsize; t++) {
        if((buff[t] != '\r') && (buff[t] != '\n')) break;
    }
    if(t) {
        chunkedsize -= t;
        memmove(buff, buff + t, chunkedsize);
    }

    return(chunkedsize);
}



int mydown_unzip(z_stream z, uint8_t *in, int inlen, uint8_t **outx, int *outxlen) {
    int         zerr,
                outsz;
    uint8_t     *out;

    if(!inlen) return(0);

    out     = *outx;
    outsz   = *outxlen;

    z.next_in   = in;
    z.avail_in  = inlen;

    for(;;) {
        z.next_out  = out   + z.total_out;
        z.avail_out = outsz - z.total_out;

        zerr = inflate(&z, Z_NO_FLUSH);

        if(zerr == Z_STREAM_END) break;
        if((zerr != Z_OK) && (zerr != Z_BUF_ERROR)) {
            fprintf(stderr, "\nError: zlib error %d\n", zerr);
            z.total_out = MYDOWN_ERROR;
            break;
        }

        if(!z.avail_in) break;

        outsz += (inlen << 1);      // inlen * 2 should be enough each time
        out = realloc(out, outsz);
        if(!out) {
            outsz       = 0;
            z.total_out = MYDOWN_ERROR;
            break;
        }
    }

    *outx    = out;
    *outxlen = outsz;
    return(z.total_out);
}



int mydown_sscanf_hex(uint8_t *data, int datalen) {
    int     i,
            ret;

    for(i = 0; i < datalen; i++) {
        if(data[i] == '\n') break;
    }
    if(i == datalen) return(MYDOWN_ERROR);

    sscanf(data, "%x", &ret);
    return(ret);
}



int mydown_timeout(int sock, int secs) {
    struct  timeval tout;
    fd_set  fdr;
    int     err;

    tout.tv_sec  = secs;
    tout.tv_usec = 0;
    FD_ZERO(&fdr);
    FD_SET(sock, &fdr);
    err = select(sock + 1, &fdr, NULL, NULL, &tout);
    if(err < 0) return(MYDOWN_ERROR); //std_err();
    if(!err) return(MYDOWN_ERROR);
    return(0);
}



int mydown_recv(int sd, uint8_t *data, int len, int timeout) {
    if(timeout) {
        if(mydown_timeout(sd, timeout) < 0) return(MYDOWN_ERROR);
    }
    return(recv(sd, data, len, 0));
}



uint8_t *mydown_showhttp80(uint16_t port) {
    static uint8_t  mini[7];

    *mini = 0;
    if(port != 80) sprintf(mini, ":%hu", port);
    return(mini);
}



void mydown_showstatus(uint32_t fsize, uint32_t ret, int timediff, int verbose) {
    uint32_t    vis;

    if(fsize) {
        if(verbose >= 0) {
            vis = (ret * 100) / fsize;
            fprintf(stderr, "%3u%%", (vis < 100) ? vis : 100);
        }
    }
    if(verbose >= 0) fprintf(stderr, "   %10u", ret);
    if(ret > 0) {
        if(verbose >= 0) {
            if(timediff) fprintf(stderr, "   %-10u", (ret * 1000) / timediff);
        }
    }
    if(verbose >= 0) fprintf(stderr, "\r");
}



uint8_t *mydown_base64_encode(uint8_t *data, int *size) {
    int         len;
    uint8_t     *buff,
                *p,
                a,
                b,
                c;
    static const uint8_t base[64] = {
        'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P',
        'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f',
        'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v',
        'w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/'
    };

    if(!size || (*size < 0)) {      // use size -1 for auto text size!
        len = strlen(data);
    } else {
        len = *size;
    }
    buff = malloc(((len / 3) << 2) + 6);
    if(!buff) return(NULL);

    p = buff;
    do {
        a     = data[0];
        b     = data[1];
        c     = data[2];
        *p++  = base[(a >> 2) & 63];
        *p++  = base[(((a &  3) << 4) | ((b >> 4) & 15)) & 63];
        *p++  = base[(((b & 15) << 2) | ((c >> 6) &  3)) & 63];
        *p++  = base[c & 63];
        data += 3;
        len  -= 3;
    } while(len > 0);
    *p = 0;

    for(; len < 0; len++) *(p + len) = '=';

    if(size) *size = p - buff;
    return(buff);
}



in_addr_t mydown_resolv(char *host) {
    struct      hostent *hp;
    in_addr_t   host_ip;
    int         i;
    dns_db_t    *dns;

    host_ip = inet_addr(host);
    if(host_ip == htonl(INADDR_NONE)) {

        for(i = 0; i < dns_db_max; i++) {           // search
            if(!stricmp(host, dns_db[i].host)) return(dns_db[i].ip);
        }

        hp = gethostbyname(host);
        if(!hp) {
            fprintf(stderr, "\nError: Unable to resolve hostname (%s)\n\n", host);
            return(0);
        }
        host_ip = *(in_addr_t *)(hp->h_addr);

        if(!dns_db_max) memset(&dns_db, 0, sizeof(dns_db));
        if(dns_db_add == MAXDNS) dns_db_add = 0;    // add
        dns = &dns_db[dns_db_add];
        if(dns->host) free(dns->host);
        dns->host = strdup(host);
        dns->ip   = host_ip;
        dns_db_add++;
        if(dns_db_max < MAXDNS) dns_db_max++;
    }
    return(host_ip);
}


